{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 对用户进行聚类\n",
    "\n",
    "数据来源于Kaggle竞赛：Event Recommendation Engine Challenge，根据\n",
    "events they’ve responded to in the past\n",
    "user demographic information\n",
    "what events they’ve seen and clicked on in our app\n",
    "用户对某个事件是否感兴趣\n",
    "\n",
    "竞赛官网：\n",
    "https://www.kaggle.com/c/event-recommendation-engine-challenge/data\n",
    "\n",
    "由于用户众多（3w+），可以对用户进行聚类\n",
    "事件描述信息在users.csv文件：共110维特征\n",
    "user_id\n",
    "locale：地区，语言\n",
    "birthyear：出身年\n",
    "gender：性别\n",
    "joinedAt：用户加入APP的时间，ISO-8601 UTC time\n",
    "location：地点\n",
    "timezone：时区\n",
    "\n",
    "作业要求：\n",
    "根据用户的属性进行聚类（KMeans聚类）\n",
    "尝试K=20， 40， 80，并计算各自CH_scores。\n",
    "\n",
    "提示：由于样本数目较多，建议使用MiniBatchKMeans。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## 导入工具包\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import metrics\n",
    "\n",
    "import time\n",
    "from matplotlib import pyplot\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>locale</th>\n",
       "      <th>birthyear</th>\n",
       "      <th>gender</th>\n",
       "      <th>joinedAt</th>\n",
       "      <th>location</th>\n",
       "      <th>timezone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3197468391</td>\n",
       "      <td>id_ID</td>\n",
       "      <td>1993</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-10-02T06:40:55.524Z</td>\n",
       "      <td>Medan  Indonesia</td>\n",
       "      <td>480.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3537982273</td>\n",
       "      <td>id_ID</td>\n",
       "      <td>1992</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-09-29T18:03:12.111Z</td>\n",
       "      <td>Medan  Indonesia</td>\n",
       "      <td>420.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>823183725</td>\n",
       "      <td>en_US</td>\n",
       "      <td>1975</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-10-06T03:14:07.149Z</td>\n",
       "      <td>Stratford  Ontario</td>\n",
       "      <td>-240.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1872223848</td>\n",
       "      <td>en_US</td>\n",
       "      <td>1991</td>\n",
       "      <td>female</td>\n",
       "      <td>2012-11-04T08:59:43.783Z</td>\n",
       "      <td>Tehran  Iran</td>\n",
       "      <td>210.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3429017717</td>\n",
       "      <td>id_ID</td>\n",
       "      <td>1995</td>\n",
       "      <td>female</td>\n",
       "      <td>2012-09-10T16:06:53.132Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>420.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      user_id locale birthyear  gender                  joinedAt  \\\n",
       "0  3197468391  id_ID      1993    male  2012-10-02T06:40:55.524Z   \n",
       "1  3537982273  id_ID      1992    male  2012-09-29T18:03:12.111Z   \n",
       "2   823183725  en_US      1975    male  2012-10-06T03:14:07.149Z   \n",
       "3  1872223848  en_US      1991  female  2012-11-04T08:59:43.783Z   \n",
       "4  3429017717  id_ID      1995  female  2012-09-10T16:06:53.132Z   \n",
       "\n",
       "             location  timezone  \n",
       "0    Medan  Indonesia     480.0  \n",
       "1    Medan  Indonesia     420.0  \n",
       "2  Stratford  Ontario    -240.0  \n",
       "3        Tehran  Iran     210.0  \n",
       "4                 NaN     420.0  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#读取数据\n",
    "import pandas as pd\n",
    "df = pd.read_csv(\"users.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38209"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_records = df.shape[0]\n",
    "n_records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 38209 entries, 0 to 38208\n",
      "Data columns (total 7 columns):\n",
      "user_id      38209 non-null int64\n",
      "locale       38209 non-null object\n",
      "birthyear    38209 non-null object\n",
      "gender       38100 non-null object\n",
      "joinedAt     38152 non-null object\n",
      "location     32745 non-null object\n",
      "timezone     37773 non-null float64\n",
      "dtypes: float64(1), int64(1), object(5)\n",
      "memory usage: 2.0+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#并统计有多少不同的users（n_users）\n",
    "def get_uniqueUsers():\n",
    "    uniqueUsers = set()\n",
    "    \n",
    "    for i in range(n_records):\n",
    "        uniqueUsers.add(df.loc[i,'user_id'])\n",
    "    \n",
    "    n_events = len(uniqueUsers)\n",
    "    return n_events\n",
    "\n",
    "n_users = get_uniqueUsers()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38209"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_users"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "记录的数目等于用户的数目，Bingo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>locale</th>\n",
       "      <th>birthyear</th>\n",
       "      <th>gender</th>\n",
       "      <th>joinedAt</th>\n",
       "      <th>timezone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id_ID</td>\n",
       "      <td>1993</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-10-02T06:40:55.524Z</td>\n",
       "      <td>480.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id_ID</td>\n",
       "      <td>1992</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-09-29T18:03:12.111Z</td>\n",
       "      <td>420.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>en_US</td>\n",
       "      <td>1975</td>\n",
       "      <td>male</td>\n",
       "      <td>2012-10-06T03:14:07.149Z</td>\n",
       "      <td>-240.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>en_US</td>\n",
       "      <td>1991</td>\n",
       "      <td>female</td>\n",
       "      <td>2012-11-04T08:59:43.783Z</td>\n",
       "      <td>210.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id_ID</td>\n",
       "      <td>1995</td>\n",
       "      <td>female</td>\n",
       "      <td>2012-09-10T16:06:53.132Z</td>\n",
       "      <td>420.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  locale birthyear  gender                  joinedAt  timezone\n",
       "0  id_ID      1993    male  2012-10-02T06:40:55.524Z     480.0\n",
       "1  id_ID      1992    male  2012-09-29T18:03:12.111Z     420.0\n",
       "2  en_US      1975    male  2012-10-06T03:14:07.149Z    -240.0\n",
       "3  en_US      1991  female  2012-11-04T08:59:43.783Z     210.0\n",
       "4  id_ID      1995  female  2012-09-10T16:06:53.132Z     420.0"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#user_id不作为聚类属性\n",
    "df = df.drop([\"user_id\"], axis=1)\n",
    "        \n",
    "#location有缺失值，粗暴抛弃\n",
    "#也可以将缺失值作为另外一类：others\n",
    "df = df.drop([\"location\"], axis=1)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#特征编码\n",
    "import datetime\n",
    "import hashlib\n",
    "import locale\n",
    "\n",
    "from collections import defaultdict\n",
    "from sklearn.preprocessing import normalize\n",
    "\n",
    "class FeatureEng:\n",
    "    def __init__(self):\n",
    "         # 载入 locales\n",
    "        self.localeIdMap = defaultdict(int)\n",
    "        for i, l in enumerate(locale.locale_alias.keys()):\n",
    "          self.localeIdMap[l] = i + 1\n",
    "        \n",
    "        # 载入 gender id 字典\n",
    "        ##缺失补0\n",
    "        self.genderIdMap = defaultdict(int, {'NaN': 0, \"male\":1, \"female\":2})\n",
    "\n",
    "  \n",
    "    def getLocaleId(self, locstr):\n",
    "        return self.localeIdMap[locstr.lower()]\n",
    "\n",
    "    def getGenderId(self, genderStr):\n",
    "        return self.genderIdMap[genderStr]\n",
    "\n",
    "    def getJoinedYearMonth(self, dateString):\n",
    "        try:\n",
    "            dttm = datetime.datetime.strptime(dateString, \"%Y-%m-%dT%H:%M:%S.%fZ\")\n",
    "            #return \"\".join([str(dttm.year), str(dttm.month)])\n",
    "            return (dttm.year-2010)*12 + dttm.month\n",
    "        except:  #缺失补0\n",
    "          return 0\n",
    "\n",
    "    def getBirthYearInt(self, birthYear):\n",
    "        #缺失补0\n",
    "        try:\n",
    "          return 0 if birthYear == \"None\" else int(birthYear)\n",
    "        except:\n",
    "          return 0\n",
    "\n",
    "    def getTimezoneInt(self, timezone):\n",
    "        try:\n",
    "          return int(timezone)\n",
    "        except:  #缺失值处理\n",
    "          return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "FE = FeatureEng()\n",
    "\n",
    "cols = ['LocaleId', 'BirthYearInt', 'GenderId', 'JoinedYearMonth', 'TimezoneInt']\n",
    "n_cols = len(cols)\n",
    "userMatrix = np.zeros((df.shape[0],n_cols), dtype=np.int)\n",
    "\n",
    "for i in range(df.shape[0]): \n",
    "    userMatrix[i, 0] = FE.getLocaleId(df.loc[i,'locale'])\n",
    "    userMatrix[i, 1] = FE.getBirthYearInt(df.loc[i,'birthyear'])\n",
    "    userMatrix[i, 2] = FE.getGenderId(df.loc[i,'gender'])\n",
    "    userMatrix[i, 3] = FE.getJoinedYearMonth(df.loc[i,'joinedAt'])\n",
    "    #userMatrix[i, 4] = FE.getCountryId(df[''])\n",
    "    userMatrix[i, 4] = FE.getTimezoneInt(df.loc[i,'timezone'])\n",
    "\n",
    "# 归一化用户矩阵\n",
    "userMatrix = normalize(userMatrix, norm=\"l1\", axis=0, copy=False)\n",
    "\n",
    "df_FE = pd.DataFrame(data=userMatrix, columns=cols)  \n",
    "#mmwrite(\"US_userMatrix\", userMatrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LocaleId</th>\n",
       "      <th>BirthYearInt</th>\n",
       "      <th>GenderId</th>\n",
       "      <th>JoinedYearMonth</th>\n",
       "      <th>TimezoneInt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000036</td>\n",
       "      <td>0.000027</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>0.000036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000036</td>\n",
       "      <td>0.000027</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>0.000031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000020</td>\n",
       "      <td>0.000027</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>-0.000018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000020</td>\n",
       "      <td>0.000027</td>\n",
       "      <td>0.000038</td>\n",
       "      <td>0.000027</td>\n",
       "      <td>0.000016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000036</td>\n",
       "      <td>0.000027</td>\n",
       "      <td>0.000038</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>0.000031</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   LocaleId  BirthYearInt  GenderId  JoinedYearMonth  TimezoneInt\n",
       "0  0.000036      0.000027  0.000019         0.000026     0.000036\n",
       "1  0.000036      0.000027  0.000019         0.000026     0.000031\n",
       "2  0.000020      0.000027  0.000019         0.000026    -0.000018\n",
       "3  0.000020      0.000027  0.000038         0.000027     0.000016\n",
       "4  0.000036      0.000027  0.000038         0.000026     0.000031"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_FE.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#缺失值填补\n",
    "#BirthYearInt\n",
    "#JoinedYearMonth     38152 non-null object\n",
    "#TimezoneInt     37773 non-null float64\n",
    "#mean_BirthYearInt = np.mean(df_FE[\"BirthYearInt\"]) \n",
    "#df_FE.loc[df_FE[\"BirthYearInt\"] == 0, \"BirthYearInt\"] = mean_BirthYearInt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MiniBatchKMeans(batch_size=100, compute_labels=True, init='k-means++',\n",
       "        init_size=None, max_iter=100, max_no_improvement=10,\n",
       "        n_clusters=100, n_init=3, random_state=None,\n",
       "        reassignment_ratio=0.01, tol=0.0, verbose=0)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.cluster import MiniBatchKMeans\n",
    "\n",
    "n_clusters = 100\n",
    "km = MiniBatchKMeans(n_clusters = n_clusters)\n",
    "km.fit(df_FE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#保存预测结果\n",
    "df_FE['cluster_100'] = km.predict(df_FE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_FE.to_csv('users_FE.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LocaleId</th>\n",
       "      <th>BirthYearInt</th>\n",
       "      <th>GenderId</th>\n",
       "      <th>JoinedYearMonth</th>\n",
       "      <th>TimezoneInt</th>\n",
       "      <th>cluster_100</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000036</td>\n",
       "      <td>0.000027</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>0.000036</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000036</td>\n",
       "      <td>0.000027</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>0.000031</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000020</td>\n",
       "      <td>0.000027</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>-0.000018</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000020</td>\n",
       "      <td>0.000027</td>\n",
       "      <td>0.000038</td>\n",
       "      <td>0.000027</td>\n",
       "      <td>0.000016</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000036</td>\n",
       "      <td>0.000027</td>\n",
       "      <td>0.000038</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>0.000031</td>\n",
       "      <td>95</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   LocaleId  BirthYearInt  GenderId  JoinedYearMonth  TimezoneInt  cluster_100\n",
       "0  0.000036      0.000027  0.000019         0.000026     0.000036           61\n",
       "1  0.000036      0.000027  0.000019         0.000026     0.000031            3\n",
       "2  0.000020      0.000027  0.000019         0.000026    -0.000018            5\n",
       "3  0.000020      0.000027  0.000038         0.000027     0.000016           28\n",
       "4  0.000036      0.000027  0.000038         0.000026     0.000031           95"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_FE.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "我猜老师这样做是想用cluster_100做训练集用来校验"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_trains=1000\n",
    "X_train=df_FE.drop('cluster_100',axis=1).values[:n_trains]\n",
    "y_train=df_FE.cluster_100.values[:n_trains]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "X_train_part, X_val, y_train_part, y_val = train_test_split(X_train,y_train, train_size = 0.8,random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(800, 5)\n",
      "(200, 5)\n"
     ]
    }
   ],
   "source": [
    "#拆分后的训练集和校验集的样本数目\n",
    "print(X_train_part.shape)\n",
    "print(X_val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 一个参数点（聚类数据为K）的模型，在校验集上评价聚类算法性能\n",
    "def K_cluster_analysis(K, X_train, y_train, X_val, y_val):\n",
    "    start = time.time()\n",
    "    \n",
    "    print(\"K-means begin with clusters: {}\".format(K));\n",
    "    \n",
    "    #K-means,在训练集上训练\n",
    "    mb_kmeans = MiniBatchKMeans(n_clusters = K)\n",
    "    mb_kmeans.fit(X_train)\n",
    "    \n",
    "    # 在训练集和测试集上测试\n",
    "    #y_train_pred = mb_kmeans.fit_predict(X_train)\n",
    "    y_val_pred = mb_kmeans.predict(X_val)\n",
    "    \n",
    "    #以前两维特征打印训练数据的分类结果\n",
    "    #plt.scatter(X_train[:, 0], X_train[:, 1], c=y_pred)\n",
    "    #plt.show()\n",
    "\n",
    "    # K值的评估标准\n",
    "    #常见的方法有轮廓系数Silhouette Coefficient和Calinski-Harabasz Index\n",
    "    #这两个分数值越大则聚类效果越好\n",
    "    #CH_score = metrics.calinski_harabaz_score(X_train,mb_kmeans.predict(X_train))\n",
    "    CH_score = metrics.silhouette_score(X_train,mb_kmeans.predict(X_train))\n",
    "    \n",
    "    #也可以在校验集上评估K\n",
    "    v_score = metrics.v_measure_score(y_val, y_val_pred)\n",
    "    \n",
    "    end = time.time()\n",
    "    print(\"CH_score: {}, time elaps:{}\".format(CH_score, int(end-start)))\n",
    "    print(\"v_score: {}\".format(v_score))\n",
    "    \n",
    "    return CH_score,v_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "K-means begin with clusters: 20\n",
      "CH_score: 0.6980983340079043, time elaps:0\n",
      "v_score: 0.7969476957867628\n",
      "K-means begin with clusters: 40\n",
      "CH_score: 0.723234330463238, time elaps:0\n",
      "v_score: 0.8603626180479834\n",
      "K-means begin with clusters: 80\n",
      "CH_score: 0.5212611277248135, time elaps:0\n",
      "v_score: 0.9299046651408659\n"
     ]
    }
   ],
   "source": [
    "Ks = [20, 40, 80]\n",
    "CH_scores = []\n",
    "v_scores = []\n",
    "for K in Ks:\n",
    "    ch,v = K_cluster_analysis(K, X_train_part, y_train_part, X_val, y_val)\n",
    "    CH_scores.append(ch)\n",
    "    v_scores.append(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XucVWXd/vHPBQgeUhEZC8EDJh5L\nMUbUSE4KoiaYmoBWYhrlT6zUfJTSNFIzMzGLVPJsKioewCMiB88ag+EBFEUoRSjxrI8KqN/fH/ea\nh+04MBuYmTV7z/V+vebF7Huvted768y+9rrXutetiMDMzKxF3gWYmVnT4EAwMzPAgWBmZhkHgpmZ\nAQ4EMzPLOBDMzAxwIJiZWcaBYGZmgAPBzMwyrfIuYHW0b98+tt5667zLMDMrKTNnznwjIirq2q6k\nAmHrrbemqqoq7zLMzEqKpH8Xs52HjMzMDHAgmJlZxoFgZmZAkYEgaYCkuZLmSTqtludHS5qVfb0o\n6Z2svaukxyXNlvSMpMEF+1wtaUHBfl3rr1tmZra66jypLKklMAboBywEZkiaGBFzqreJiBMLtj8B\n2C17+CHwg4h4SdLmwExJkyLinez5UyJifD31xczM1kIxRwjdgXkRMT8ilgHjgEGr2H4ocCNARLwY\nES9l3y8CXgfqvPTJzMwaXzGB0BF4teDxwqztCyRtBXQGptbyXHegNfByQfM52VDSaEltiq7azMzq\nXTGBoFraVrbu5hBgfER8+rkXkDoA1wFHR8RnWfNIYAdgd6AdcGqtP1waLqlKUtWSJUuKKNeaunff\nhXvugUsugfffz7saM6tWzMS0hcAWBY87AYtWsu0Q4PjCBkkbAXcDp0fEE9XtEbE4+3appKuAX9T2\nghExFhgLUFlZ6QWgS9CSJfDII/Dgg/DQQ/D00/BZ9rHguuvg3nth443zrdHMiguEGUAXSZ2B10hv\n+kfU3EjS9sAmwOMFba2B24FrI+KWGtt3iIjFkgQcDDy3xr2wJuW119Ibf/XXnOzyg/XWg732gl//\nGnr2hNdfh+9/H/bdFyZNgnbt8q3brLmrMxAi4hNJI4BJQEvgyoiYLWkUUBURE7NNhwLjIqLwU/zh\nQE9gU0nDsrZhETELuF5SBWlIahbwk3rpkTWqCJg///MBMH9+em7DDeFb30pv+r16Qbdu0Lr15/ff\nYAM49FDYZx+YPBnat2/8PphZos+/fzdtlZWV4XsZ5SsCnn9+xfDPQw/BomwAcdNN0yf/6q9dd4WW\nLet+zUmT4OCD4atfhSlT4Mtfbtg+mDU3kmZGRGVd25XUze2s8X36aRrzr37zf/hheOON9Nzmm6dP\n/tUBsMMO0GIN5r7vtx/cfTccdBD07p1CYfPN67UbZlYEB4J9zrJlUFW1IgAefRTeey89t8028O1v\npzf/Xr2gc2dQbdegrYG+feG+++CAA9JrT50KW2xR935mVn8cCM3chx/Ck0+uGAJ64gn46KP03E47\nwRFHpADYe2/o1Klha9l7b7j/fhgwYEUoePkLs8bjQGhm3n0XHntsxRHAjBmwfHka6unaFX784xQA\n3/oWVOQwp3yvvdKQUf/+qY6pU2HbbRu/DrPmyIFQ5t54I437VwfArFlpDkCrVrD77nDSSenT+De/\n2XTmAlRWpiDYd99U25Qp6fyEmTUsB0KZWdkcgHXXTZ++zzgjffLec09Yf/18a12Vrl1h+vR0OWr1\nieadd867KrPy5kAoYRGwYEF6468+B1DbHICePdOn7ppzAJq6r30t9atv3xQKDzyQLmU1s4bhQCgh\n1XMACo8AXnstPbfppumk7AknpADYZZc0LFTqdthhRSj06ZMmr3XrlndVZuWpDN4yyteq5gB06PD5\nOQA77rhmcwBKQZcuqf99+qQhpPvuS0NeZla/HAhNyLJlMHPmiuGflc0B6NkzPa6vOQCloHPn9N+k\nb990BdI996QhMTOrPw6EHFXPAag+Anj88RVzAHbcsXHnAJSCLbdMYbnPPmmuwl13pXMLZlY/HAiN\n6L330qf+mnMApHRVzfDhK+YAbLZZ3tU2TR07rrj66IADYMIE6Ncv76rMyoMDoQEVMwegZ880B6Bt\n27yrLR1f+UoKhX790v2PbrsthYOZrR0HQj167bXPB8Ds2am95hyAPfZIt322NVdRkSav9e+f7pR6\n883pXzNbcw6ENVQ4B6D66+VstegNN4QePeDII1fMAWjjFaPrXbt2aW7CgAHw3e/CDTekf81szRQV\nCJIGAH8iLZBzeUScV+P50UCf7OH6wGYR0TZ77ijg9Oy5syPimqy9G3A1sB5wD/CzaMKLM0TACy98\nfh2A6jkA7dqlN/7jj1+xDkA5zAEoBW3bphviHXggDBmSzskc8YX1/MysGHW+bUlqCYwB+pHWV54h\naWJEzKneJiJOLNj+BGC37Pt2wJlAJRDAzGzft4FLgOHAE6RAGADcW0/9WmuffgrPPPP5I4DmOAeg\nFGy0UVqXeeBA+N730uW7w4blXZVZ6Snmc2x3YF5EzAeQNA4YBMxZyfZDSSEAsB8wOSLeyvadDAyQ\nNB3YKCIez9qvJa2rnFsgVM8BqH7zf+SRFXMAOndOn0CrQ6C5zQEoBV/6UroM9eCD4eij0//P4cPz\nrsqstBQTCB2BVwseLwT2qG1DSVsBnYGpq9i3Y/a1sJb2RlPXHIChQ1fMAfBCLaVh/fVh4sS0RvOP\nf5xCYcSIvKsyKx3FBEJtn4VXNtY/BBgfEZ/WsW/RrylpOGloiS233HLVla7Ce+99fh2Af/zDcwDK\n0brrpstQBw9O93Vatixd3mtmdSsmEBYChZ+ROwGLVrLtEOD4Gvv2rrHv9Ky9U432Wl8zIsYCYwEq\nKyvX6KTzkUfCuHEr5gBUVsKJJ65YB8BzAMpLmzZwyy3p//vJJ8PSpTByZN5VmTV9xQTCDKCLpM7A\na6Q3/S9cxyFpe2AT4PGC5knAuZI2yR73B0ZGxFuS3pe0J/Ak8APgz2vejVX75jfTqlvV6wB4DkD5\nW2eddBlq69bwy1+mI4Vf/9rnfsxWpc5AiIhPJI0gvbm3BK6MiNmSRgFVETEx23QoMK7w0tHsjf+3\npFABGFV9ghk4jhWXnd5LA55QPv74urex8tOqFVxzTQqHs85KRwrnnONQMFsZNeFL/7+gsrIyqqqq\n8i7DSsxnn8Fxx8HYsel8wgUXOBSseZE0MyIq69rO06es7LVoAZdemoaPLrwwDR9dfLFDwawmB4I1\nC1IKgTZt4I9/TKFwySWeUGhWyIFgzYYEf/hDCoVzz02hcPnl0LJl3pWZNQ0OBGtWJDj77DR8dNZZ\nKRSuucb3njIDB4I1QxKceeaKS1KXL4frr09XI5k1Zw4Ea7ZGjkzDRyefnEJh3DjfptyaN59Ss2bt\npJPgz3+GO+6AQw6Bjz/OuyKz/DgQrNkbMQIuu2zFLbQ//DDviszy4UAwI93c8Mor0wpsBx4IH3yQ\nd0Vmjc+BYJYZNgyuuy7dDXfAgBXrYZg1Fw4EswLVd8Z98kno3x/eeSfviswajwPBrIbvfhfGj4en\nnoJ994W33qp7H7Ny4EAwq8WgQenKo+eegz59YMmSvCsya3gOBLOVOOCAtCTniy9C797wn//kXZFZ\nw3IgmK1C//5wzz3wr3+lFfZeey3viswajgPBrA59+sCkSbB4cQqFV17JuyKzhlFUIEgaIGmupHmS\nTlvJNodLmiNptqQbsrY+kmYVfH0s6eDsuaslLSh4rmv9dcusfn3rWzB5MrzxRgqFBQvyrsis/tUZ\nCJJaAmOA/YGdgKGSdqqxTRdgJNAjInYGfg4QEdMiomtEdAX6Ah8C9xfsekr18xExq156ZNZA9tgD\npkyBd99N63O/9FLeFZnVr2KOELoD8yJifkQsA8YBg2ps8yNgTES8DRARr9fyOocB90aEbwxgJatb\nN5g2Ld3zqFcveOGFvCsyqz/FBEJH4NWCxwuztkLbAdtJelTSE5IG1PI6Q4Aba7SdI+kZSaMl1Xqf\nSUnDJVVJqlria/+sCdh1V5g+Pa3V3KtXujTVrBwUEwi1rTwbNR63AroAvYGhwOWS2v7fC0gdgK8D\nkwr2GQnsAOwOtANOre2HR8TYiKiMiMqKiooiyjVreDvvDA8+mBbW6d0bZnnA08pAMYGwENii4HEn\nYFEt20yIiOURsQCYSwqIaocDt0fE8uqGiFgcyVLgKtLQlFnJ2H77FArrrw99+0JVVd4Vma2dYgJh\nBtBFUmdJrUlDPxNrbHMH0AdAUnvSENL8gueHUmO4KDtqQJKAgwEfeFvJ2XbbdDO8jTeGffaBxx/P\nuyKzNVdnIETEJ8AI0nDP88DNETFb0ihJA7PNJgFvSpoDTCNdPfQmgKStSUcYD9Z46eslPQs8C7QH\nzl777pg1vq23TqGw2WZpItvDD+ddkdmaUUTN0wFNV2VlZVT5uNyaqEWL0lHCK6/AnXemYSSzpkDS\nzIiorGs7z1Q2qyebb56uPtpmm7TIzqRJde5i1qQ4EMzq0Ze/nOYp7LBDWo7zrrvyrsiseA4Es3rW\nvn2a0bzLLnDIIXD77XlXZFYcB4JZA2jXLq3P3K1bWnDnppvyrsisbg4Esway8cZw//3wzW/CEUfA\n3/+ed0Vmq+ZAMGtAG24I996bZjP/4Adw5ZV5V2S2cg4Eswa2wQbp5HK/fnDMMXDppXlXZFY7B4JZ\nI1hvPZgwIV2OetxxcPHFeVdk9kUOBLNGsu66cNtt8J3vwM9+BhdckHdFZp/nQDBrRK1bpyuOBg+G\nU06Bc87JuyKzFVrlXYBZc7POOumKo9at4fTTYdkyOOssUG03mjdrRA4Esxy0agVXXZXCYdQoWLoU\nfvc7h4Lly4FglpOWLeFvf0tHCr//fTpS+OMfHQqWHweCWY5atIC//jWFwujRKRQuvji1mzU2B4JZ\nziS46CJo0wb+8Ic0fHTZZQ4Fa3xF/cpJGiBprqR5kk5byTaHS5ojabakGwraP5U0K/uaWNDeWdKT\nkl6SdFO2GptZsySlYaNf/Qouvxx++EP49NO8q7Lmps4jBEktgTFAP9LayTMkTYyIOQXbdAFGAj0i\n4m1JmxW8xEcR0bWWl/49MDoixkm6FDgGuGQt+mJW0iQ4++x0pPDrX6fho2uvTSegzRpDMUcI3YF5\nETE/IpYB44BBNbb5ETAmIt4GiIjXV/WC2TrKfYHxWdM1pHWVzZq9M86A886DG2+EIUNg+fK8K7Lm\nophA6Ai8WvB4YdZWaDtgO0mPSnpC0oCC59aVVJW1V7/pbwq8k63XvLLXNGu2Tj0VLrwQbr0VDjss\nnVcwa2jFHIzWdhFczYWYWwFdgN5AJ+BhSV+LiHeALSNikaRtgKmSngXeK+I10w+XhgPDAbbccssi\nyjUrDyeemIaPjj8+3e7i1lvTPZHMGkoxRwgLgS0KHncCFtWyzYSIWB4RC4C5pIAgIhZl/84HpgO7\nAW8AbSW1WsVrku03NiIqI6KyoqKiqE6ZlYv/9//SXIX77ktLcn74Yd4VWTkrJhBmAF2yq4JaA0OA\niTW2uQPoAyCpPWkIab6kTSS1KWjvAcyJiACmAYdl+x8FTFjbzpiVo2OPTbOap06FAw6ADz7IuyIr\nV3UGQjbOPwKYBDwP3BwRsyWNkjQw22wS8KakOaQ3+lMi4k1gR6BK0tNZ+3kFVyedCpwkaR7pnMIV\n9dkxs3Jy1FHp/kePPAIDBsB7tQ26mq0lpQ/rpaGysjKqqqryLsMsN7femq48+sY30jDSJpvkXZGV\nAkkzI6Kyru08F9KshBx6aAqFWbNgn33gzTfzrsjKiQPBrMQMHAh33AFz5kCfPvD6Kmf9mBXPgWBW\ngvbfP63TPG8e9O4NixfnXZGVAweCWYnad1+491545ZUUCq+9lndFVuocCGYlrFcvmDQpHSH07An/\n/nfeFVkpcyCYlbgePeCBB9IJ5p49Yf78vCuyUuVAMCsD3buniWsffJBC4cUX867ISpEDwaxMfOMb\nMG1aum12r17w/PN5V2SlxoFgVkZ22QWmT0/f9+oFzz6bazlWYhwIZmVmp53gwQfTOs19+sA//5l3\nRVYqHAhmZWi77VIobLAB9O0L//hH3hVZKXAgmJWpr34VHnoo3e9o333hscfyrsiaOgeCWRnbaqsU\nCl/5CvTvn743WxkHglmZ69QpDR9tuWW6dfaUKXlXZE2VA8GsGejQIV19tO228O1vp1tnm9XkQDBr\nJjbbLM1T2HFHGDQI7rwz74qsqSkqECQNkDRX0jxJp61km8MlzZE0W9INWVtXSY9nbc9IGlyw/dWS\nFkialX11rZ8umdnKbLppGjLadVc45JC0toJZtToDQVJLYAywP7ATMFTSTjW26QKMBHpExM7Az7On\nPgR+kLUNAC6S1LZg11Miomv2NWvtu2NmddlkE5g8Od3uYvBgGDcu74qsqSjmCKE7MC8i5kfEMmAc\nMKjGNj8CxkTE2wAR8Xr274sR8VL2/SLgdaCivoo3szWz8cbpPEKPHnDkkXDttXlXZE1BMYHQEXi1\n4PHCrK3QdsB2kh6V9ISkATVfRFJ3oDXwckHzOdlQ0mhJbWr74ZKGS6qSVLVkyZIiyjWzYmy4Idxz\nT5rNPGwYXHFF3hVZ3ooJBNXSFjUetwK6AL2BocDlhUNDkjoA1wFHR8RnWfNIYAdgd6AdcGptPzwi\nxkZEZURUVlT44MKsPm2wQTq5vN9+cOyx8Ne/5l2R5amYQFgIbFHwuBOwqJZtJkTE8ohYAMwlBQSS\nNgLuBk6PiCeqd4iIxZEsBa4iDU2ZWSNbb720RvNBB8Hxx8NFF+VdkeWlmECYAXSR1FlSa2AIMLHG\nNncAfQAktScNIc3Ptr8duDYibincITtqQJKAg4Hn1qYjZrbm2rSB8ePh0EPhxBPh/PPzrsjyUGcg\nRMQnwAhgEvA8cHNEzJY0StLAbLNJwJuS5gDTSFcPvQkcDvQEhtVyeen1kp4FngXaA2fXa8/MbLW0\nbp2uOBoyBE49FX7727wrssamiJqnA5quysrKqKqqyrsMs7L26adw9NFw3XVw+ukwahSotjOJVjIk\nzYyIyrq2a9UYxZhZ6WjZEq66Kh0xnH12WoHtvPMcCs2BA8HMvqBlSxg7NoXC+efD0qUwerRDodw5\nEMysVi1awJgx6YTzRRelI4W//CW1W3lyIJjZSklw4YUpFH7/+xQKl12WjiCs/DgQzGyVJPjd79Lw\n0W9/m0Lhyiuhld89yo7/l5pZnaR0tVHr1nDGGbB8ebr/0Trr5F2Z1ScHgpkV7fTT0/DR//xPOlK4\n8cYUElYefHrIzFbLKaekk8y33QaHHZauQLLy4EAws9X2s5+lG+HdeWdafe2jj/KuyOqDA8HM1shx\nx8Hll8P996d1mv/3f/OuyNaWA8HM1tgxx8A118D06bD//vD++3lXZGvDgWBma+X734cbboDHHkvr\nKrz7bt4V2ZpyIJjZWhs8GG6+GaqqoF8/ePvtvCuyNeFAMLN6ccghcOut8PTT0LcvvPFG3hXZ6nIg\nmFm9OeggmDABXnghrdX8+ut5V2Sro6hAkDRA0lxJ8ySdtpJtDpc0R9JsSTcUtB8l6aXs66iC9m6S\nns1e8+Js5TQzK3EDBsBdd8HLL0Pv3rB4cd4VWbHqDARJLYExwP7ATsBQSTvV2KYLMBLoERE7Az/P\n2tsBZwJ7kNZMPlPSJtlulwDDSWsvdwEG1EeHzCx/++wD990Hr74KvXrBwoV5V2TFKOYIoTswLyLm\nR8QyYBwwqMY2PwLGRMTbABFRfaC4HzA5It7KnpsMDMjWU94oIh6PtGTbtaR1lc2sTPTsmeYo/Pe/\n6ft//SvviqwuxQRCR+DVgscLs7ZC2wHbSXpU0hOSBtSxb8fs+1W9ppmVuL32ggceSFcd9eqVhpGs\n6SomEGob26+5EHMr0rBPb2AocLmktqvYt5jXTD9cGi6pSlLVkiVLiijXzJqS3XeHqVPTTOaePWHu\n3LwrspUpJhAWAlsUPO4ELKplmwkRsTwiFgBzSQGxsn0XZt+v6jUBiIixEVEZEZUVFRVFlGtmTc1u\nu8G0afDJJ+lIYc6cvCuy2hQTCDOALpI6S2oNDAEm1tjmDqAPgKT2pCGk+cAkoL+kTbKTyf2BSRGx\nGHhf0p7Z1UU/ACbUS4/MrEn6+tfTLS5atEhXHz3zTN4VWU11BkJEfAKMIL25Pw/cHBGzJY2SNDDb\nbBLwpqQ5wDTglIh4MyLeAn5LCpUZwKisDeA44HJgHvAycG899svMmqAdd4QHH0xrKvTpA089lXdF\nVkjpIp/SUFlZGVVVVXmXYWZraf78NJv5nXdg0iTYY4+8KypvkmZGRGVd23mmspk1um22SUcKm26a\n7n306KN5V2TgQDCznGy1FTz0EHTokO6SOn163hWZA8HMctOxYzpS2GorOOCANGfB8uNAMLNcfeUr\n6eigS5e08to99+RdUfPlQDCz3FVUpMlrO+8MBx+c7phqjc+BYGZNwqabwpQpaRLbYYfB+PF5V9T8\nOBDMrMlo2xYmT06XoQ4ZkpbmtMbjQDCzJmWjjdKts/feG773Pbjmmrwraj4cCGbW5HzpS3D33Wld\nhaOPhr/9Le+KmgcHgpk1SeuvD3femVZgGz4cxozJu6Ly50AwsyZr3XXh9tth0CAYMQJGj867ovLm\nQDCzJq1NG7jllnTl0UknwXnn5V1R+WqVdwFmZnVZZx248UZo3RpGjoRly+CMM0C1LbVla8yBYGYl\noVUruPbaFA5nnglLl8LZZzsU6pMDwcxKRsuWcOWV6Ujh3HPTkcL55zsU6osDwcxKSosWcOmlKRQu\nuCCFwkUXORTqQ1EnlSUNkDRX0jxJp9Xy/DBJSyTNyr6Ozdr7FLTNkvSxpIOz566WtKDgua712zUz\nK1ctWsCf/5xOMl98MRx3HHz2Wd5Vlb46jxAktQTGAP2AhcAMSRMjouYy2TdFxIjChoiYBnTNXqcd\nabnM+ws2OSUifMcSM1ttUjpCaN06XXm0bFmawNayZd6Vla5ihoy6A/MiYj6ApHHAIKBmINTlMODe\niPhwNfczM6uVlM4ltGkDv/lNCoWrr04noG31FTNk1BF4teDxwqytpkMlPSNpvKQtanl+CHBjjbZz\nsn1GS2pT2w+XNFxSlaSqJUuWFFGumTUnEpx1FpxzDlx/PRx5JCxfnndVpamYQKjtVE3UeHwnsHVE\n7AI8AHzudlSSOgBfByYVNI8EdgB2B9oBp9b2wyNibERURkRlRUVFEeWaWXP0y1+mIaSbb4bBg9PR\ngq2eYgJhIVD4ib8TsKhwg4h4MyKWZg//BnSr8RqHA7dHxPKCfRZHshS4ijQ0ZWa2xk4+OZ1kvv12\nOOQQ+PjjvCsqLcUEwgygi6TOklqThn4mFm6QHQFUGwg8X+M1hlJjuKh6H0kCDgaeW73Szcy+6IQT\n0mWpd9+d7oH0oc9aFq3OUy8R8YmkEaThnpbAlRExW9IooCoiJgI/lTQQ+AR4CxhWvb+krUlHGA/W\neOnrJVWQhqRmAT9Z696YmQE//nG6+uiYY9I6zXfeCRtskHdVTZ8iap4OaLoqKyujqqoq7zLMrET8\n/e9w1FHQo0c6Ythww7wryoekmRFRWdd2vtupmZWt730v3RTvscegf3945528K2raHAhmVtYOPxzG\nj4eZM2HffeGtt/KuqOlyIJhZ2Tv44HTl0bPPQt++4ClNtXMgmFmzcOCB6eTy3LnQpw/89795V9T0\nOBDMrNno3z+dXF6wAHr3hkWL6tylWXEgmFmz0rcv3HcfLFwIvXrBq6/WvU9z4UAws2Zn771h8mR4\n/XXo2TMdMZgDwcyaqT33hClT4N1305HCvHl5V5Q/B4KZNVuVlTB1Knz0UQqFF17Iu6J8ORDMrFnr\n2hWmTYNPP00nmp9rxndVcyCYWbP3ta/B9Olpac4+feDpp/OuKB8OBDMzYIcd4KGHYL31Uig0x9um\nORDMzDLbbgsPPggbbwz77ANPPJF3RY3LgWBmVqBz5xQKFRXQrx888kjeFTUeB4KZWQ1bbpmGjzp2\nhP32Syedm4OiAkHSAElzJc2TdFotzw+TtETSrOzr2ILnPi1on1jQ3lnSk5JeknRTthqbmVmTsPnm\n6Uihc2c44AC4//68K2p4dQaCpJbAGGB/YCdgqKSdatn0pojomn1dXtD+UUH7wIL23wOjI6IL8DZw\nzJp3w8ys/n35y+noYPvt4aCD0n2QylkxRwjdgXkRMT8ilgHjgEFr80OzdZT7AuOzpmtI6yqbmTUp\nFRVp8trXvw7f+Q7ccUfeFTWcYgKhI1B4+6eFWVtNh0p6RtJ4SVsUtK8rqUrSE5Kq3/Q3Bd6JiE/q\neE0zs9y1awcPPADdusF3vwu33JJ3RQ2jmEBQLW01F2K+E9g6InYBHiB94q+2ZbaW5xHARZK+WuRr\nph8uDc8CpWqJV7Uws5y0bZvOI+y5JwwZAtdfn3dF9a+YQFgIFH7i7wR87i7iEfFmRCzNHv4N6Fbw\n3KLs3/nAdGA34A2graRWK3vNgv3HRkRlRFRWVFQUUa6ZWcPYcMN06+xeveD734errsq7ovpVTCDM\nALpkVwW1BoYAEws3kNSh4OFA4PmsfRNJbbLv2wM9gDkREcA04LBsn6OACWvTETOzxrDBBnDXXWl9\n5h/+EC67LO+K6k+rujaIiE8kjQAmAS2BKyNitqRRQFVETAR+Kmkg8AnwFjAs231H4DJJn5HC57yI\nmJM9dyowTtLZwD+BK+qxX2ZmDWb99WHiRDjsMPjJT2DZMjjhhLyrWntKH9ZLQ2VlZVQ1xxuMmFmT\ntGwZDB6crjy64AI4+eS8K6qdpJnZudxV8kxlM7M11Lo13HwzHH44/OIXcO65eVe0duocMjIzs5Vb\nZ510xdE668CvfpWOGs48E1TbtZRNnAPBzGwttWoF11yTQuE3v0mhcM45pRcKDgQzs3rQsiVccQW0\naQO/+x0sXZrOK5RSKDgQzMzqSYsWcMkl6dzChRemI4U//Sm1lwIHgplZPZJSCLRpk44Qli6FSy8t\njVBwIJiZ1TMJzj8/HSmce246UrjiijSs1JQ5EMzMGoAEZ5+djhTOPBOWL08nnls14XfdJlyamVlp\nk+DXv05HCiNHplCovkS1KXK5ajFqAAAHSElEQVQgmJk1sNNOS0cKJ52Uho9uuik9bmpK4DSHmVnp\nO/FE+MtfYMIEOOQQ+PjjvCv6IgeCmVkjOf74dHfUe+9NS3J++GHeFX2eA8HMrBENHw5XXglTpsCB\nB8IHH+Rd0QoOBDOzRjZsGPz97/DwwzBgALz3Xt4VJQ4EM7McHHEEjBsHTz4J/fvDO+/kXZEDwcws\nN4cdBuPHw1NPwT77wJtv5ltPUYEgaYCkuZLmSTqtlueHSVoiaVb2dWzW3lXS45JmS3pG0uCCfa6W\ntKBgn6711y0zs9IwaFBaYGf2bOjbF5Ysya+WOgNBUktgDLA/sBMwVNJOtWx6U0R0zb4uz9o+BH4Q\nETsDA4CLJLUt2OeUgn1mrV1XzMxK0wEHwJ13wksvQe/e8J//5FNHMUcI3YF5ETE/IpYB44BBxbx4\nRLwYES9l3y8CXgcq1rRYM7Ny1a8f3HMP/Pvf0KsXvPZa49dQTCB0BF4teLwwa6vp0GxYaLykLWo+\nKak70Bp4uaD5nGyf0ZJqnbcnabikKklVS/I8ljIza2C9e8OkSbB4cQqFV15p3J9fTCDUtrxD1Hh8\nJ7B1ROwCPABc87kXkDoA1wFHR8RnWfNIYAdgd6AdcGptPzwixkZEZURUVlT44MLMyluPHjB5Mrzx\nBvTsCfPnN97PLiYQFgKFn/g7AYsKN4iINyNiafbwb0C36uckbQTcDZweEU8U7LM4kqXAVaShKTOz\nZm+PPdLEtfffT0cKL73UOD+3mECYAXSR1FlSa2AIMLFwg+wIoNpA4PmsvTVwO3BtRNxS2z6SBBwM\nPLemnTAzKzfdusHUqemeR716wYsvNvzPrDMQIuITYAQwifRGf3NEzJY0StLAbLOfZpeWPg38FBiW\ntR8O9ASG1XJ56fWSngWeBdoDZ9dbr8zMysCuu8KDD8Iuu0D79g3/8xRR83RA01VZWRlVVVV5l2Fm\nVlIkzYyIyrq280xlMzMDHAhmZpZxIJiZGeBAMDOzjAPBzMwAB4KZmWUcCGZmBjgQzMwsU1IT0yQt\nAf69hru3B96ox3LyVC59KZd+gPvSVJVLX9a2H1tFRJ13By2pQFgbkqqKmalXCsqlL+XSD3Bfmqpy\n6Utj9cNDRmZmBjgQzMws05wCYWzeBdSjculLufQD3Jemqlz60ij9aDbnEMzMbNWa0xGCmZmtQtkF\ngqQtJE2T9Hy2aM/PsvZ2kiZLein7d5O8a62LpHUl/UPS01lffpO1d5b0ZNaXm7KV6UqCpJaS/inp\nruxxSfZF0r8kPZst+lSVtZXi71hbSeMlvZD9zexVov3YvmARrlmS3pP081LsC4CkE7O/+eck3Zi9\nFzT430rZBQLwCXByROwI7AkcL2kn4DRgSkR0AaZkj5u6pUDfiNgV6AoMkLQn8HtgdNaXt4Fjcqxx\ndf2MbInVTCn3pU9EdC24HLAUf8f+BNwXETsAu5L+35RcPyJibvb/oitpTfcPScv3llxfJHUkrTxZ\nGRFfA1qSli5u+L+ViCjrL2AC0A+YC3TI2joAc/OubTX7sT7wFLAHaYJKq6x9L2BS3vUV2YdOpD/K\nvsBdgEq4L/8C2tdoK6nfMWAjYAHZucRS7Uct/eoPPFqqfQE6Aq8C7YBW2d/Kfo3xt1KORwj/R9LW\nwG7Ak8CXI2IxQPbvZvlVVrxsiGUW8DowGXgZeCfSWtcAC0m/QKXgIuB/gM+yx5tSun0J4H5JMyUN\nz9pK7XdsG2AJcFU2jHe5pA0ovX7UNAS4Mfu+5PoSEa8BFwCvAIuBd4GZNMLfStkGgqQvAbcCP4+I\n9/KuZ01FxKeRDoM7Ad2BHWvbrHGrWn2Svg28HhEzC5tr2bTJ9yXTIyK+AexPGpbsmXdBa6AV8A3g\nkojYDfhfSmBIZVWycfWBwC1517KmsvMcg4DOwObABqTfs5rq/W+lLANB0jqkMLg+Im7Lmv8rqUP2\nfAfSJ+6SERHvANNJ50XaSmqVPdUJWJRXXauhBzBQ0r+AcaRho4sozb4QEYuyf18njVV3p/R+xxYC\nCyPiyezxeFJAlFo/Cu0PPBUR/80el2Jf9gUWRMSSiFgO3AZ8k0b4Wym7QJAk4Arg+Yi4sOCpicBR\n2fdHkc4tNGmSKiS1zb5fj/SL8jwwDTgs26wk+hIRIyOiU0RsTTqknxoRR1KCfZG0gaQNq78njVk/\nR4n9jkXEf4BXJW2fNe0DzKHE+lHDUFYMF0Fp9uUVYE9J62fvZ9X/Xxr8b6XsJqZJ+hbwMPAsK8aq\nf0k6j3AzsCXpP/h3I+KtXIoskqRdgGtIVxm0AG6OiFGStiF9ym4H/BP4XkQsza/S1SOpN/CLiPh2\nKfYlq/n27GEr4IaIOEfSppTe71hX4HKgNTAfOJrsd40S6geApPVJJ2O3iYh3s7aS+38CkF1iPph0\n1eQ/gWNJ5wwa9G+l7ALBzMzWTNkNGZmZ2ZpxIJiZGeBAMDOzjAPBzMwAB4KZmWUcCGZmBjgQzMws\n40AwMzMA/j8pF5BW8kVCfgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x21565c346a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(Ks, np.array(CH_scores), 'b-')\n",
    "pyplot.savefig('CH_scores.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XmcjXX/x/HXx9gJWSqZ3CiFSouJ\nFltkSaId2VvUfafutvsudxkzpLSnUt0qS4SkkqJoMRghI1SUEsVE2cluZj6/P+Z03/Ob233PkZm5\nzpnzfj4eHnOu61zXnPfXnPOea77nnOuYuyMiIrGhWNABRESk8Kj0RURiiEpfRCSGqPRFRGKISl9E\nJIao9EVEYohKX0Qkhqj0RURiiEpfRCSGFA86QG5Vq1b1WrVqBR1DRCSqLFmyZIu7V8tru4gr/Vq1\napGWlhZ0DBGRqGJmP4WznaZ3RERiiEpfRCSGqPRFRGKISl9EJIao9EVEYohKX0Qkhqj0RURiiEpf\nRCRgWZ7Fmyve5OUlLxf4ban0RUQC4u5M/XYq5/zzHK6bch2jl42moD+3XKUvIlLI3J3p300n4eUE\nrnzjSvYd2sfrV73OvL7zMLMCve2IOw2DiEhR5e58tOYjEmcnsujnRdQ5tg5jOo+he8PuFC9WOHWs\n0hcRKQSz184mMSWR1HWp1KxYk5cvf5neZ/WmRFyJQs2h0hcRKUCp61IZOHsgKT+mUOOYGrzQ4QVu\nPPdGSsaVDCSPSl9EpAAsTF9I4uxEPlrzEceXO57h7YfTr1E/ShcvHWgulb6ISD5K25DGoJRBzPh+\nBtXKVuOJNk/w5/P+TNkSZYOOBqj0RUTyxbJflpGUksS7q96lcpnKPNL6Efo37k/5kuWDjvb/qPRF\nRI7Cik0rGJQyiLe+eYtKpSsx5OIh3NHkDiqUqhB0tMNS6YuI/AGrtqwiaU4Sb3z9BuVLliexeSJ3\nXXAXlUpXCjra/6TSFxE5Aqu3rWbI3CGM/3I8ZYqX4f6m93PPBfdQpWyVoKOFJazSN7P2wHAgDnjF\n3Yfluv5PwCigGrAN6OHu6WZ2NvAiUAHIBIa6+xv5mF9EpFD8uONHhswZwtjlYykZV5K7z7+bv1/0\nd6qVy/OzyCNKnqVvZnHACKANkA4sNrNp7r4yx2ZPAK+5+1gzawU8AvQE9gK93P17MzsRWGJmM919\nR76PRESkAKzfuZ6h84by6tJXibM4+jfuz/1N7+eE8icEHe0PCedIvzGw2t3XAJjZJKAzkLP0GwB3\nhS7PBqYCuPt3v2/g7hvMbBPZfw2o9EUkom34bQOPzHuEkV+MxN3pd24//tHsH9SoUCPoaEclnNKv\nAazPsZwONMm1zXLgarKngK4EjjGzKu6+9fcNzKwxUBL44agSi4gUoF93/8qj8x/lxbQXycjKoO/Z\nfXmw+YPUrFgz6Gj5IpzSP9wp33Kf+/Ne4Hkz6wPMBX4GMv71DcyqA+OA3u6e9R83YNYP6AdQs2bR\n+I8VkeiyZe8WHp//OM8vfp4DGQfodVYvHmz+IHWOrRN0tHwVTumnAyflWI4HNuTcwN03AFcBmFl5\n4Gp33xlargBMBx5094WHuwF3HwmMBEhISCjYk0mLiOSwbd82nlrwFMMXDWfPwT10b9idxOaJ1K1S\nN+hoBSKc0l8M1DWz2mQfwXcFrs+5gZlVBbaFjuIHkP1KHsysJPAO2U/yvpmfwUVEjsbO/Tt5ZuEz\nPLXwKXYd2MV1p19HUosk6lerH3S0ApVn6bt7hpn1B2aS/ZLNUe6+wswGA2nuPg1oCTxiZk729M5t\nod2vA5oDVUJTPwB93H1Z/g5DRCQ8vx34jWcXPcsTC55gx/4dXFX/KpJaJHHm8WcGHa1QWEF/NNeR\nSkhI8LS0tKBjiEgRs+fgHkYsHsFj8x9j676tXH7q5SS3TOac6ucEHS1fmNkSd0/Iazu9I1dEirR9\nh/bxUtpLDJs/jE17NtH+lPYkt0ymcY3GQUcLhEpfRIqkAxkHePmLl3l43sNs3L2R1rVbM/jiwVx4\n0oVBRwuUSl9EipSDmQcZvXQ0D817iPRd6TSr2YyJV0+kRa0WQUeLCCp9ESkSDmUeYtyX4xgydwg/\n7viRC+IvYEznMbSq3Qqzw73dKDap9EUkqmVmZTLhqwkkz0nmh+0/kHBiAi9e9iLtTm6nsj8Mlb6I\nRKUsz2LyiskkpSSxausqzj7hbKZ1nUbHUzuq7P8Hlb6IRJUsz+Kdb95hUMogVmxewRnHncFb173F\nFfWuoJgVCzpexFPpi0hUcHemrZrGoJRBLP91OfWq1mPS1ZO49vRrVfZHQKUvIhHN3flg9Qckzk5k\nycYlnFL5FMZdOY5uZ3Qjrlhc0PGijkpfRCKSu/Pxmo9JTElkYfpCalWqxahOo+h5Vk+KF1N1/VH6\nnxORiJPyYwqJsxOZt24eJ1U4iZEdR9L77N6UjCsZdLSop9IXkYgxf918ElMS+XTtp5x4zImM6DCC\nG8+5kVLFSwUdrchQ6YtI4BalLyIxJZFZP8zi+HLH80y7Z+jXqB9lSpQJOlqRo9IXkcB8sfELEmcn\nMv376VQtW5XH2zzOX877C2VLlA06WpGl0heRQvflr18yKGUQU7+dyrGlj+XhVg/Tv3F/jil1TNDR\nijyVvogUmpWbV5KUksSbK9+kYqmKJLdM5q9N/krF0hWDjhYzVPoiUuBWbVnF4LmDmfjVRMqVLMeD\nzR7k7gvu5tgyxwYdLeao9EWkwPyw7QeGzB3CuC/HUbp4af5+0d+598J7qVq2atDRYpZKX0Ty3U87\nfuKhuQ8xetloSsSV4M4md3Jf0/s4rtxxQUeLeSp9Eck36bvSeXjew7zyxSuYGX857y8MaDqA6sdU\nDzqahKj0ReSobfxtI4+kPsI/l/wTd+emc2/iH83+QXyF+KCjSS5hnZrOzNqb2SozW21m9x/m+j+Z\n2Sdm9qWZpZhZfI7repvZ96F/vfMzvIgEa9OeTdwz8x7qPFuHFxa/QK+Gvfju9u944bIXVPgRKs8j\nfTOLA0YAbYB0YLGZTXP3lTk2ewJ4zd3Hmlkr4BGgp5lVBgYBCYADS0L7bs/vgYhI4dm6dyuPf/Y4\nz33+HPsz9tOzYU8GNh/IyZVPDjqa5CGc6Z3GwGp3XwNgZpOAzkDO0m8A3BW6PBuYGrrcDvjI3beF\n9v0IaA9MPProIlLYtu/bzlMLnuKZRc+w5+Aeup3ZjUEtBnFqlVODjiZhCqf0awDrcyynA01ybbMc\nuBoYDlwJHGNmVf7LvjX+cFoRCcTO/TsZvmg4Ty14ip0HdnJtg2tJaplEg2oNgo4mRyic0j/ch016\nruV7gefNrA8wF/gZyAhzX8ysH9APoGbNmmFEEpHCsPvgbp5b9ByPf/Y42/dv54p6V5DcMpmGxzcM\nOpr8QeGUfjpwUo7leGBDzg3cfQNwFYCZlQeudvedZpYOtMy1b0ruG3D3kcBIgISEhP/4pSAihWvv\nob28sPgFHp3/KFv2buGyupeR3DKZRic2CjqaHKVwSn8xUNfMapN9BN8VuD7nBmZWFdjm7lnAAGBU\n6KqZwMNm9vt7rduGrheRCLQ/Yz//TPsnj6Q+wq97fqXdye1IbplMk/jcM7oSrfIsfXfPMLP+ZBd4\nHDDK3VeY2WAgzd2nkX00/4iZOdnTO7eF9t1mZkPI/sUBMPj3J3VFJHIcyDjAK1+8wsOpD7Phtw20\nqt2KKS2n0LRm06CjST4z98iaTUlISPC0tLSgY4jEhEOZhxi9bDQPzX2I9bvW07RmU4ZcPISWtVoG\nHU2OkJktcfeEvLbTO3JFYlBGVgbjlo9jyNwhrN2xliY1mvBqp1e5pM4lmB3u9RdSVKj0RWJIZlYm\nE7+eSPKcZFZvW02j6o14vsPzXHrKpSr7GKHSF4kBWZ7FmyveJGlOEt9u+Zazjj+LqV2m0um0Tir7\nGKPSFynCsjyLqd9OZVDKIL7e9DWnVzudKddO4cr6V1LMwjr1lhQxKn2RIsjdef+790lMSWTZL8s4\nrcppTLx6Itc2uJa4YnFBx5MAqfRFihB3Z+YPM0mcncjiDYs5+diTee2K1+h2ZjeKF9PDXVT6IkWC\nu/PJ2k9InJ3IgvQF1KpUi1c7vUrPhj0pEVci6HgSQVT6IlFu7k9zGTh7IHN/mkt8hXheuuwl+p7T\nl5JxJYOOJhFIpS8SpT5b/xmJsxP5ZO0nVC9fnecufY6bz72ZUsVLBR1NIphKXyTKfP7z5wxKGcSH\nqz/kuHLH8VTbp7g14VbKlCgTdDSJAip9kSixdONSBqUM4r3v3qNKmSo8esmj3HbebZQrWS7oaBJF\nVPoiEe6rX79iUMog3vn2HY4tfSxDWw3l9sa3c0ypY4KOJlFIpS8Sob7Z/A1Jc5KYvGIyFUpVIKlF\nEneefycVS1cMOppEMZW+SIT5fuv3JM9JZsJXEyhXshwPNHuAuy+4m8plKgcdTYoAlb5IhFizfQ1D\n5g5h3PJxlCpeir9d+Df+dtHfqFq2atDRpAhR6YsE7KcdPzF03lBGLxtN8WLFuaPJHdx30X0cX/74\noKNJEaTSFwnIz7t+5uF5D/PyFy9jZtza6FYGNBvAicecGHQ0KcJU+iKF7JfdvzAsdRgvpb1Epmdy\n4zk38kCzBzip4klBR5MYoNIXKSSb92zmsfmPMWLxCA5mHqT3Wb0Z2GIgtSrVCjqaxBCVvkgB27p3\nK08ueJJnFz3Lvox99GjYg4HNB3JK5VOCjiYxSKUvUkB27N/B0wue5umFT7P74G66nNGFQS0GUa9q\nvaCjSQwLq/TNrD0wHIgDXnH3YbmurwmMBSqFtrnf3WeYWQngFeDc0G295u6P5GN+kYiz68Auhi8c\nzpMLnmTngZ1c0+AaBrUYxBnHnRF0NJG8S9/M4oARQBsgHVhsZtPcfWWOzR4EJrv7i2bWAJgB1AKu\nBUq5+5lmVhZYaWYT3f3HfB6HSOB2H9zN858/z+OfPc62fdvofFpnklomcfYJZwcdTeRfwjnSbwys\ndvc1AGY2CegM5Cx9ByqELlcENuRYX87MigNlgIPArnzILRIx9h7ay4uLX+TR+Y+yee9mOtTtQHLL\nZBJOTAg6msh/CKf0awDrcyynA01ybZMEzDKz24FywCWh9VPI/gWxESgL3OXu23LfgJn1A/oB1KxZ\n8wjiiwRnf8Z+Ri4ZySOpj/DL7l9oU6cNgy8ezPnx5wcdTeS/Cqf07TDrPNdyN2CMuz9pZhcA48zs\nDLL/SsgETgSOBeaZ2ce//9Xwr2/mPhIYCZCQkJD7e4tElAMZBxi1dBRD5w3l599+pmWtlky+ZjLN\n/tQs6GgieQqn9NOBnO8aieff0ze/uxFoD+DuC8ysNFAVuB740N0PAZvMbD6QAKxBJMocyjzE2OVj\nGTJ3COt2ruOiky5i3JXjuLj2xUFHEwlbsTC2WQzUNbPaZlYS6ApMy7XNOqA1gJnVB0oDm0PrW1m2\ncsD5wLf5FV6kMGRkZTB22VjqjajHze/dzAnlT2Bmj5nM6ztPhS9RJ88jfXfPMLP+wEyyX445yt1X\nmNlgIM3dpwH3AC+b2V1kT/30cXc3sxHAaOBrsqeJRrv7lwU1GJH8lJmVyaSvJ5E8J5nvt33PudXP\n5f1u79OhbgfMDjfrKRL5wnqdvrvPIPtlmDnXJea4vBK46DD77Sb7ZZsiUSPLs5iycgpJKUl8s+Ub\nGh7fkHe6vEPn0zqr7CXq6R25IiHuztRvpzIoZRBfbfqK+lXrM/mayVzd4GqKWTgzoSKRT6UvMc/d\nmf79dBJnJ7L0l6WcWuVUXr/qdbqc3oW4YnFBxxPJVyp9iVnuzqwfZpGYksjnP39OnWPrMKbzGLo3\n7E7xYnpoSNGke7bEpE/Xfkri7ETmr5/Pnyr+iVcuf4VeZ/WiRFyJoKOJFCiVvsSUeT/NIzElkZQf\nU6hxTA1evOxFbjjnBkrGlQw6mkihUOlLTFiwfgGJKYl8vOZjTih/As+2f5abG91M6eKlg44mUqhU\n+lKkpW1II3F2Ih+s/oBqZavxZNsnuTXhVsqWKBt0NJFAqPSlSFr2yzIGpQxi2qppVC5TmWGth3Fb\n49soX7J80NFEAqXSlyLl601fk5SSxFvfvEWl0pUYcvEQ7mhyBxVKVch7Z5EYoNKXIuHbLd+SPCeZ\nN75+g/Ily5PYPJG7LriLSqUrBR1NJKKo9CWqrd62msFzBvP6V69TpngZBjQdwD0X3kPlMpWDjiYS\nkVT6EpXWbl/LQ3MfYuzysZSMK8k9F9zD3y78G9XKVQs6mkhEU+lLVFm/cz0PzX2IUctGEWdx9G/c\nn/ub3s8J5U8IOppIVFDpS1TY8NsGHp73MC9/8TLuzi2NbmFA0wHUqFAj6GgiUUWlLxHt192/Mix1\nGC8teYmMrAxuOPsGHmj+ADUr6rOURf4Ilb5EpC17t/DY/Md4/vPnOZh5kF5n9eLB5g9S59g6QUcT\niWoqfYko2/Zt48nPnuTZz59lz8E9dG/YncTmidStUjfoaCJFgkpfIsKO/Tt4ZuEzPL3waXYd2EWX\n07swqMUg6lerH3Q0kSJFpS+B+u3Abzy76FmeWPAEO/bv4Kr6V5HUIokzjz8z6GgiRZJKXwKx5+Ae\nRiwewWPzH2Prvq1cfurlJLdM5pzq5wQdTaRIU+lLodp3aB8vpb3EsPnD2LRnE5eecinJLZM5r8Z5\nQUcTiQlhfdqzmbU3s1VmttrM7j/M9TXNbLaZLTWzL82sQ47rGprZAjNbYWZfmZlOYB6D9mfs57lF\nz3Hysydz96y7aXh8Q+bfMJ8Z3Weo8EUKUZ5H+mYWB4wA2gDpwGIzm+buK3Ns9iAw2d1fNLMGwAyg\nlpkVB8YDPd19uZlVAQ7l+ygkYh3MPMiopaMYOm8o6bvSaf6n5ky8eiItarUIOppITApneqcxsNrd\n1wCY2SSgM5Cz9B34/dy1FYENocttgS/dfTmAu2/Nj9ASHbbv207HiR35bP1nXBB/AWM6j6FV7VaY\nWdDRRGJWOKVfA1ifYzkdaJJrmyRglpndDpQDLgmtPxVwM5sJVAMmuftjuW/AzPoB/QBq1tQ7LYuC\njb9tpN34dqzauooJV02g6xldVfYiESCcOf3DPVI913I3YIy7xwMdgHFmVozsXypNge6hr1eaWev/\n+GbuI909wd0TqlXTWRKj3drta2k2uhlrtq9hxvUz6HZmNxW+SIQIp/TTgZNyLMfz7+mb390ITAZw\n9wVAaaBqaN857r7F3feSPdd/7tGGlsj19aavuWjURWzfv51Pen1C6zr/8TteRAIUTukvBuqaWW0z\nKwl0Babl2mYd0BrAzOqTXfqbgZlAQzMrG3pStwX//7kAKUIWpS+i+ejmmBlz+8ylSXzuWUARCVqe\npe/uGUB/sgv8G7JfpbPCzAabWafQZvcAN5vZcmAi0MezbQeeIvsXxzLgC3efXhADkWB9vOZjWr/W\nmsplKpPaN5XTjzs96Egichjmnnt6PlgJCQmelpYWdAw5Am9/8zbd3upGvar1mNljpj7QRCQAZrbE\n3RPy2i6sN2eJ/Dejlo7i2jevJeHEBFJ6p6jwRSKcSl/+sCc/e5Ibp91I25PbMqvHLI4tc2zQkUQk\nDyp9OWLuzgOfPMC9H91Ll9O78G7XdylXslzQsUQkDDrhmhyRzKxM+s/oz0tLXuKWRrcwosMI4orF\nBR1LRMKk0pewHcw8SK93evHGijcY0HQAQ1sN1ZuuRKKMSl/CsvfQXq6ZfA0frP6Axy55jL9d9Leg\nI4nIH6DSlzzt2L+DjhM6siB9AS9f/jI3nXtT0JFE5A9S6cv/9OvuX2k3vh0rN6/kjWve4JoG1wQd\nSUSOgkpf/qsfd/xIm3Ft2PjbRqZfP502J7cJOpKIHCWVvhzWys0raTuuLXsP7eXjXh9zfvz5QUcS\nkXyg0pf/sPjnxbR/vT2l4koxt+9czjjujKAjiUg+0Zuz5P/5dO2ntHqtFZVKVyL1hlQVvkgRo9KX\nf5n67VQuff1SalWqRWrfVOocWyfoSCKSz1T6AsCYZWO4evLVnFv9XOb0mUP1Y6oHHUlECoBKX3hm\n4TP0fbcvrWu35uOeH1O5TOWgI4lIAVHpxzB3Z+CnA7lr5l1c0+Aa3uv2nk6cJlLE6dU7MSrLs7jj\ngzsYsXgEN51zEy91fEknThOJASr9GHQo8xB93u3DhK8m8PcL/86wS4bpxGkiMUKlH2P2HtrLdW9e\nx/TvpzOs9TDua3pf0JFEpBCp9GPIzv07uXzi5aSuS+WfHf9Jv0b9go4kIoUsrCdyzay9ma0ys9Vm\ndv9hrq9pZrPNbKmZfWlmHQ5z/W4zuze/gsuR2bRnEy3HtmRh+kImXTNJhS8So/I80jezOGAE0AZI\nBxab2TR3X5ljsweBye7+opk1AGYAtXJc/zTwQb6lliPy046faDu+Lem70nmv23u0O6Vd0JFEJCDh\nTO80Bla7+xoAM5sEdAZylr4DFUKXKwIbfr/CzK4A1gB78iOwHJlvt3xLm3Ft2H1wNx/1/IgLT7ow\n6EgiEqBwpndqAOtzLKeH1uWUBPQws3Syj/JvBzCzcsB9QPJRJ5UjlrYhjWajm3Eo8xBz+sxR4YtI\nWKV/uNfyea7lbsAYd48HOgDjzKwY2WX/tLvv/p83YNbPzNLMLG3z5s3h5JY8pPyYwsVjL6Z8yfKk\n3pBKw+MbBh1JRCJAONM76cBJOZbjyTF9E3Ij0B7A3ReYWWmgKtAEuMbMHgMqAVlmtt/dn8+5s7uP\nBEYCJCQk5P6FIkdo2qppXPfmdZxS+RRm9phJjQq5/zATkVgVTukvBuqaWW3gZ6ArcH2ubdYBrYEx\nZlYfKA1sdvdmv29gZknA7tyFL/lr3PJx9H23L41ObMSM62dQpWyVoCOJSATJc3rH3TOA/sBM4Buy\nX6WzwswGm1mn0Gb3ADeb2XJgItDH3XXEXsieXfQsvab2omWtlnzS6xMVvoj8B4u0bk5ISPC0tLSg\nY0QVdyd5TjLJc5K5qv5VTLhqAqWKlwo6logUIjNb4u4JeW2nd+RGuSzP4s4P7+S5z5+j79l9GXn5\nSIoX049VRA5P7RDFDmUe4oZpNzD+y/Hcc8E9PN7mcZ04TUT+J5V+lNp3aB9dpnThve/eY2iroQxo\nOkCFLyJ5UulHoV0HdtFpYifm/jSXFzq8wJ/P+3PQkUQkSqj0o8zmPZtp/3p7vvz1SyZcPYGuZ3QN\nOpKIRBGVfhRZv3M9bca1Yd3Odbzb9V061O2Q904iIjmo9KPEqi2raDOuDbsO7GJWz1k0rdk06Egi\nEoVU+lHgi41f0H58e8yMlD4pnH3C2UFHEpEoFdaHqEhw5v40l5ZjWlK2RFlS+6aq8EXkqKj0I9j7\n371Pu/HtiK8QT+oNqdStUjfoSCIS5VT6Eer1L1/niklXcOZxZzK371ziK8QHHUlEigCVfgR6/vPn\n6fFOD1rUasEnvT6hatmqQUcSkSJCpR9B3J0hc4Zw+we3c0W9K5h+/XSOKXVM0LFEpAjRq3ciRJZn\ncffMuxm+aDi9z+rNK51e0YnTRCTfqVUiQEZWBjdNu4mxy8dyZ5M7ebLdkxQz/REmIvlPpR+w/Rn7\n6TqlK++uepchFw/hgWYP6MRpIlJgVPoB+u3Ab3Se1JnZP87m+Uuf57bGtwUdSUSKOJV+QLbs3cKl\nr1/K0o1LGX/leLo37B50JBGJASr9AKTvSqftuLas3bGWqV2n0vHUjkFHEpEYodIvZN9v/Z5Lxl3C\njv07mNljJs3/1DzoSCISQ1T6hWjZL8toN74d7s7s3rM5t/q5QUcSkRgT1usCzay9ma0ys9Vmdv9h\nrq9pZrPNbKmZfWlmHULr25jZEjP7KvS1VX4PIFqkrkulxZgWlIorReoNqSp8EQlEnqVvZnHACOBS\noAHQzcwa5NrsQWCyu58DdAVeCK3fAlzu7mcCvYFx+RU8msz4fgZtx7XlxGNOZP4N8zm1yqlBRxKR\nGBXOkX5jYLW7r3H3g8AkoHOubRyoELpcEdgA4O5L3X1DaP0KoLSZlTr62NFj4lcT6TypMw2qNWBu\nn7mcVPGkoCOJSAwLp/RrAOtzLKeH1uWUBPQws3RgBnD7Yb7P1cBSdz/wB3JGpRcXv0j3t7tz0UkX\n8WnvT6lWrlrQkUQkxoVT+od7e6jnWu4GjHH3eKADMM7s3+cRMLPTgUeBWw57A2b9zCzNzNI2b94c\nXvII5u4MnTuUv8z4Cx1P7cgH3T+gQqkKee8oIlLAwin9dCDnnEQ8oembHG4EJgO4+wKgNFAVwMzi\ngXeAXu7+w+FuwN1HunuCuydUqxbdR8Puzr2z7uXB2Q/Ss2FP3rruLcqUKBN0LBERILzSXwzUNbPa\nZlaS7Cdqp+XaZh3QGsDM6pNd+pvNrBIwHRjg7vPzL3ZkysjK4MZpN/LUwqe4vfHtjLliDCXiSgQd\nS0TkX/IsfXfPAPoDM4FvyH6VzgozG2xmnUKb3QPcbGbLgYlAH3f30H6nAAPNbFno33EFMpKA7c/Y\nz3VvXsfoZaNJapHE8PbDdaZMEYk4lt3NkSMhIcHT0tKCjnFEfjvwG1e+cSWfrP2E4e2Hc0eTO4KO\nJCIxxsyWuHtCXtvpHblHaeverXSY0IElG5bw2hWv0fOsnkFHEhH5r1T6R+HnXT/Tdnxbftj2A293\neZtOp3XKeycRkQCp9P+g1dtW02ZcG7bu3cqHPT6kZa2WQUcSEcmTSv8PWP7LctqNb0emZzK792wa\nndgo6EgiImHRy0uO0GfrP6Pl2JaUiCvBvL7zVPgiElVU+kfgw9Ufcslrl3BcueOYf8N86lWtF3Qk\nEZEjotIP0xtfv0GniZ2oV7Ue8/rOo2bFmkFHEhE5Yir9MIxcMpJub3Xj/Pjzmd17NseVK5LvLxOR\nGKDSz8Ow1GHc8v4tdKjbgZk9ZlKxdMWgI4mI/GF69c5/4e7c9/F9PP7Z41x/5vWM6azz6IhI9FPp\nH0ZmVia3vH8Lry59ldvOu41nL31W59ERkSJBpZ/LgYwD9HinB1NWTmFg84Ekt0zG7HAfKSAiEn1U\n+jnsPribq964io/WfMTT7Z7mzvPvDDqSiEi+UumHbNu3jcsmXMbnP3/O6M6j6XN2n6AjiYjkO5U+\nsPG3jbQd35bvtn7HW9e9xRXAhiPBAAAHj0lEQVT1rgg6kohIgYj50v9h2w+0GdeGzXs380H3D2hV\nu1XQkURECkxMl/5Xv35F2/FtOZR5iE97fcp5Nc4LOpKISIGK2dchLli/gOZjmhNnccztO1eFLyIx\nISZLf9YPs7hk3CVULVuV1BtSaVCtQdCRREQKRcyV/pSVU+g4oSN1K9cltW8qtSrVCjqSiEihianS\nf+WLV+gypQuNazQmpU8Kx5c/PuhIIiKFKqzSN7P2ZrbKzFab2f2Hub6mmc02s6Vm9qWZdchx3YDQ\nfqvMrF1+hj8Sj81/jJvfu5l2J7djVs9ZVCpdKagoIiKBybP0zSwOGAFcCjQAuplZ7knwB4HJ7n4O\n0BV4IbRvg9Dy6UB74IXQ9ys07s79H9/PfR/fR9czujK161TKlihbmBFERCJGOEf6jYHV7r7G3Q8C\nk4DOubZxoELockVgQ+hyZ2CSux9w97XA6tD3KxSZWZnc+v6tPDr/UW5tdCvjrxxPybiShXXzIiIR\nJ5zSrwGsz7GcHlqXUxLQw8zSgRnA7UewL2bWz8zSzCxt8+bNYUb/3w5mHuT6t69n5BcjeaDZA7xw\n2QvEFSvUPzJERCJOOKV/uFNMeq7lbsAYd48HOgDjzKxYmPvi7iPdPcHdE6pVqxZGpP9tz8E9dJrY\nickrJvNEmyd4qNVDOlOmiAjhvSM3HTgpx3I8/56++d2NZM/Z4+4LzKw0UDXMffPV9n3b6TixIwvT\nF/Jqp1e54ZwbCvLmRESiSjhH+ouBumZW28xKkv3E7LRc26wDWgOYWX2gNLA5tF1XMytlZrWBusDn\n+RU+t192/0KLMS1I25DGm9e+qcIXEcklzyN9d88ws/7ATCAOGOXuK8xsMJDm7tOAe4CXzewusqdv\n+ri7AyvMbDKwEsgAbnP3zIIYyPqd67l47MX8svsXpl8/nUvqXFIQNyMiEtUsu5sjR0JCgqelpR3x\nfnsO7qHLlC4MbD6QJvFNCiCZiEjkMrMl7p6Q13ZF5iyb5UqW4/3r3w86hohIRIup0zCIiMQ6lb6I\nSAxR6YuIxBCVvohIDFHpi4jEEJW+iEgMUemLiMQQlb6ISAyJuHfkmtlm4Kej+BZVgS35FCdIRWUc\noLFEqqIylqIyDji6sfzJ3fM8TXHElf7RMrO0cN6KHOmKyjhAY4lURWUsRWUcUDhj0fSOiEgMUemL\niMSQolj6I4MOkE+KyjhAY4lURWUsRWUcUAhjKXJz+iIi8t8VxSN9ERH5L6K29M3sJDObbWbfmNkK\nM/traH1lM/vIzL4PfT026Kx5MbPSZva5mS0PjSU5tL62mS0KjeWN0MdVRjwzizOzpWb2fmg5Wsfx\no5l9ZWbLzCwttC7q7l8AZlbJzKaY2behx8wF0TgWMzst9PP4/d8uM7szSsdyV+jx/rWZTQz1QIE/\nVqK29Mn++MV73L0+cD5wm5k1AO4HPnH3usAnoeVIdwBo5e5nAWcD7c3sfOBR4OnQWLaT/QH00eCv\nwDc5lqN1HAAXu/vZOV5GF433L4DhwIfuXg84i+yfT9SNxd1XhX4eZwONgL3AO0TZWMysBnAHkODu\nZ5D9UbRdKYzHirsXiX/Au0AbYBVQPbSuOrAq6GxHOI6ywBdAE7LfpFE8tP4CYGbQ+cLIH0/2g64V\n8D5g0TiOUNYfgaq51kXd/QuoAKwl9BxeNI8lV/62wPxoHAtQA1gPVCb7EwzfB9oVxmMlmo/0/8XM\nagHnAIuA4919I0Do63HBJQtfaEpkGbAJ+Aj4Adjh7hmhTdLJvqNEumeAvwNZoeUqROc4AByYZWZL\nzKxfaF003r/qAJuB0aFpt1fMrBzROZacugITQ5ejaizu/jPwBLAO2AjsBJZQCI+VqC99MysPvAXc\n6e67gs7zR7l7pmf/yRoPNAbqH26zwk11ZMysI7DJ3ZfkXH2YTSN6HDlc5O7nApeSPX3YPOhAf1Bx\n4FzgRXc/B9hDhE9/5CU0190JeDPoLH9E6DmHzkBt4ESgHNn3s9zy/bES1aVvZiXILvzX3f3t0Opf\nzax66PrqZB85Rw133wGkkP08RSUz+/3D6+OBDUHlCtNFQCcz+xGYRPYUzzNE3zgAcPcNoa+byJ43\nbkx03r/SgXR3XxRankL2L4FoHMvvLgW+cPdfQ8vRNpZLgLXuvtndDwFvAxdSCI+VqC19MzPgVeAb\nd38qx1XTgN6hy73JnuuPaGZWzcwqhS6XIfsO8Q0wG7gmtFnEj8XdB7h7vLvXIvtP70/dvTtRNg4A\nMytnZsf8fpns+eOvicL7l7v/Aqw3s9NCq1oDK4nCseTQjX9P7UD0jWUdcL6ZlQ112e8/kwJ/rETt\nm7PMrCkwD/iKf88f/4Psef3JQE2y/2OvdfdtgYQMk5k1BMaS/Qx+MWCyuw82szpkHzFXBpYCPdz9\nQHBJw2dmLYF73b1jNI4jlPmd0GJxYIK7DzWzKkTZ/QvAzM4GXgFKAmuAvoTua0TfWMqS/SRoHXff\nGVoXdT+X0Euzu5D9SsSlwE1kz+EX6GMlaktfRESOXNRO74iIyJFT6YuIxBCVvohIDFHpi4jEEJW+\niEgMUemLiMQQlb6ISAxR6YuIxJD/A1lOzR+zGdAjAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2156460f0b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(Ks, np.array(v_scores), 'g-')\n",
    "pyplot.savefig('V_scores.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
